/**
 * 
 */
package de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.model;
import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap;
import it.unimi.dsi.fastutil.longs.Long2DoubleOpenHashMap;
import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
import de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.data.Word;
import de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.data.WordCorpus;
import de.uni_leipzig.informatik.asv.util.IntegerCombine;
/**
 * 
 * @author Christoph Teichmann
 * created Jun 1, 2014 2:46:04 PM
 * @version 0.1
 */
public class MultinomialWithLexicon extends GenericModel
{
	/* (non-Javadoc)
	 * @see de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.model.Model#makeLogLikelihood()
	 */
	@Override
	public double makeLogLikelihood()
	{
		double ret = 0.0;
		for(Word w : this.getData())
		{ret += this.makeLogProbability(w, this.getTag(w));}
		return ret;
	}
	/**
	 * @param data
	 * @param featureSmoothing
	 * @param numberOfLabels
	 * @param tagTypeSmoothing
	 * @param tagToWordSmoothing
	 * @param transitionSmoothing
	 */
	public MultinomialWithLexicon(WordCorpus data, double featureSmoothing,
			int numberOfLabels, double tagTypeSmoothing,
			double tagToWordSmoothing, double transitionSmoothing)
	{
		super(data);
		if(featureSmoothing <= 0.0 || transitionSmoothing <= 0.0 
												|| tagToWordSmoothing <= 0.0 || transitionSmoothing <= 0.0)
		{throw new IllegalArgumentException("smoothing is only possible with positive values.");}
		this.featureSmoothing = featureSmoothing;
		if(numberOfLabels < 2)
		{throw new IllegalArgumentException("learning needs at least 2 labels");}
		this.numberOfLabels = numberOfLabels;
		this.tagTypeSmoothing = tagTypeSmoothing;
		this.tagToWordSmoothing = tagToWordSmoothing;
		this.transitionSmoothing = transitionSmoothing;
	}
	/**
	 * features counts
	 */
	private final Long2ObjectOpenHashMap<Int2DoubleOpenHashMap> l2t2f = new Long2ObjectOpenHashMap<>();
	/**
	 * feature group counts
	 */
	private final Long2ObjectOpenHashMap<Int2DoubleOpenHashMap> l2t2fg = new Long2ObjectOpenHashMap<>();
	/**
	 * 
	 */
	private final double featureSmoothing;
	/**
	 * 
	 */
	private final int numberOfLabels;
	/**
	 * 
	 *
	 * @param language
	 * @param tag
	 * @param feature
	 * @param featureGroup
	 * @param amount
	 * @return
	 */
	private final double getFeatureWeight(int language, int tag,
													int feature, int featureGroup, double amount)
	{
		Int2DoubleOpenHashMap m = getFeatureMap(language, tag);
		double seen = m.get(feature);
		m = getFeatureGroupMap(language, tag);
		double div = m.get(featureGroup);
		return amount*(Math.log(seen)-Math.log(div));
	}
	/**
	 * 
	 *
	 * @param language
	 * @param tag
	 * @param feature
	 * @param featureGroup
	 * @param amount
	 */
	private void addFeatureObservation(int language, int tag, int feature,int featureGroup,
																					double amount)
	{
		Int2DoubleOpenHashMap fc = this.getFeatureMap(language, tag);
		fc.addTo(feature, amount);
		fc = this.getFeatureGroupMap(language, tag);
		fc.addTo(featureGroup, amount);
	}
	@Override
	protected void initializeFeature(Word w, int feature, int featureGroup,
			double value, int id, int language) 
	{
		for(int tag=0;tag<this.getNumberOfLabels(w);++tag)
		{
			Int2DoubleOpenHashMap m = this.getFeatureMap(language, tag);
			double seen = m.get(feature);
			if(seen < featureSmoothing)
			{this.addFeatureObservation(language, tag, feature, featureGroup, this.featureSmoothing);}
			else
			{return;}
		}
	}
	/**
	 * 
	 * @param language
	 * @param tag
	 * @return
	 */
	private Int2DoubleOpenHashMap getFeatureGroupMap(int language, int tag)
	{return getSecure(l2t2fg, IntegerCombine.combine(language, tag));}
	/**
	 * 
	 * @param language
	 * @param tag
	 * @return
	 */
	private Int2DoubleOpenHashMap getFeatureMap(int language, int tag)
	{return getSecure(l2t2f, IntegerCombine.combine(language, tag));}
	/* (non-Javadoc)
	 * @see de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.model.Model#getNumberOfLabels(de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.data.Word)
	 */
	@Override
	public int getNumberOfLabels(Word w)
	{return this.numberOfLabels;}
	/* (non-Javadoc)
	 * @see de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.model.GenericModel#addFeature(de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.data.Word, int, int, int, int, int, double, double)
	 */
	@Override
	protected void addFeature(Word w, int choice, int label, int group, int id,
			int language, double amount, double multiplier)
	{this.addFeatureObservation(language, choice, label, group, amount*multiplier);}
	/* (non-Javadoc)
	 * @see de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.model.GenericModel#makeFeatureLogProb(de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.data.Word, int, int, int, double, int, int)
	 */
	@Override
	protected double makeFeatureLogProb(Word w, int choice, int label,
			int group, double value, int id, int language)
	{return this.getFeatureWeight(language, choice, label, group, value);}
	/* (non-Javadoc)
	 * @see de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.model.GenericModel#getLengthGuess()
	 */
	@Override
	protected int getLengthGuess()
	{return this.numberOfLabels;}
	/* (non-Javadoc)
	 * @see de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.model.GenericModel#getCurrentMaxNumberOfLabelsForLanguage(int)
	 */
	@Override
	protected int getCurrentMaxNumberOfLabelsForLanguage(int lang)
	{return this.numberOfLabels;}
	/**
	 *
	 * @param map
	 * @param code
	 * @return
	 */
	private Int2DoubleOpenHashMap getSecure(
			Long2ObjectOpenHashMap<Int2DoubleOpenHashMap> map, long code)
	{
		Int2DoubleOpenHashMap m = map.get(code);
		if(m == null)
		{
			m = new Int2DoubleOpenHashMap();
			map.put(code, m);
		}
		return m;
	}
	/**
	 * language tag type count
	 */
	private final Long2DoubleOpenHashMap l2lex = new Long2DoubleOpenHashMap();
	/**
	 * language type normalizer
	 */
	private final Int2DoubleOpenHashMap llex = new Int2DoubleOpenHashMap();
	/**
	 * 
	 */
	private final double tagTypeSmoothing;
	/**
	 * 
	 *
	 * @param language
	 * @param tag
	 */
	private double getTypeWeight(int language, int tag)
	{
		double w = getLanguageTagWeight(language, tag);
		double n = getLanguageNorm(language);
		return Math.log(w)-Math.log(n);
	}
	/**
	 * 
	 *
	 * @param language
	 * @param tag
	 * @param word
	 * @param amount
	 * @return
	 */
	private double getWordWeight(int language, int tag, int word, double amount)
	{
		Int2DoubleOpenHashMap m = getTagWordWeightMap(language, tag);
		double seen = m.get(word);
		double norm = getLanguageTagNorm(language, tag);
		return amount*(Math.log(seen)-Math.log(norm));
	}
	/**
	 */
	private double getLanguageTagWeight(int language, int tag)
	{return this.l2lex.get(IntegerCombine.combine(language, tag));}
	/**
	 * 
	 * @param language
	 * @return
	 */
	private double getLanguageNorm(int language)
	{return this.llex.get(language);}
	/**
	 * 
	 * @param language
	 * @param tag
	 * @return
	 */
	private double getLanguageTagNorm(int language, int tag)
	{return this.l2t.get(IntegerCombine.combine(language, tag));}
	/**
	 * 
	 * @param language
	 * @param tag
	 * @return
	 */
	private Int2DoubleOpenHashMap getTagWordWeightMap(int language, int tag)
	{return this.getSecure(l2t2w, IntegerCombine.combine(language, tag));}
	/**
	 * 
	 *
	 * @param language
	 * @param tag
	 * @param word
	 * @param amount
	 */
	private void addWordObservation(int language, int tag, int word, double amount)
	{
		long code = IntegerCombine.combine(language, tag);
		Int2DoubleOpenHashMap m = this.getTagWordWeightMap(language, tag);
		m.addTo(word, amount);
		l2t.addTo(code, amount);
	}
	/**
	 * 
	 *
	 * @param language
	 * @param tag
	 * @param positive
	 */
	private void addTypeEntry(int language, int tag, double amount)
	{
		long code = IntegerCombine.combine(language, tag);
		l2lex.addTo(code, amount);
		llex.addTo(language, amount);
	}
	@Override
	protected void initializeWord(Word w, int id, int language, double a) 
	{
		int word = w.getIdentity();
		for(int tag = 0;tag<this.getNumberOfLabels(w);++tag)
		{
			double d = getLanguageTagWeight(language, tag);
			if(d < tagTypeSmoothing)
			{this.addTypeEntry(language, tag, this.tagTypeSmoothing);}
			Int2DoubleOpenHashMap m = this.getTagWordWeightMap(language, tag);
			double seen = m.get(word);
			if(seen < this.tagToWordSmoothing)
			{this.addWordObservation(language, tag, word, this.tagToWordSmoothing);}
		}
	}
	/* (non-Javadoc)
	 * @see de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.model.GenericModel#addWord(de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.data.Word, int, int, int, double, double)
	 */
	@Override
	protected void addWord(Word w, int id, int choice, int language,
			double amount, double multiplier)
	{
		this.addTypeEntry(language, choice, multiplier);
		this.addWordObservation(language, choice, id, amount*multiplier);
	}
	/* (non-Javadoc)
	 * @see de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.model.GenericModel#makeWordLogProb(de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.data.Word, int, int, double)
	 */
	@Override
	protected double makeWordLogProb(Word w, int id, int language, int choice, double amount)
	{
		double ret = this.getTypeWeight(language, choice);
		ret += this.getWordWeight(language, choice, id, amount);
		return ret;
	}
	/**
	 * language tag word count
	 */
	private final Long2ObjectOpenHashMap<Int2DoubleOpenHashMap> l2t2w = new Long2ObjectOpenHashMap<>();
	/**
	 * 
	 * language tag count
	 */
	private final Long2DoubleOpenHashMap l2t = new Long2DoubleOpenHashMap();
	/**
	 * 
	 */
	private final double tagToWordSmoothing;
	/**
	 * counts the number of times a certain transition was seen
	 */
	private final Long2ObjectOpenHashMap<Long2ObjectOpenHashMap<Int2DoubleOpenHashMap>> l2t2ty2l2t = 
																			new Long2ObjectOpenHashMap<>();
	/**
	 * 	counts the number of times a certain transition was seen without regard for the tag used						
	 */
	private final Long2ObjectOpenHashMap<Long2DoubleOpenHashMap> l2t2ty2l =
																			new Long2ObjectOpenHashMap<>();
	/**
	 * 
	 */
	private final double transitionSmoothing;
	/**
	 * 																	
	 *
	 * @param language
	 * @param tag
	 * @param label
	 * @param otherLanguage
	 * @param otherTag
	 * @param amount
	 * @return
	 */
	private double getTransitionWeight(int language, int tag, int label, int otherLanguage,
																	int otherTag, double amount)
	{
		Int2DoubleOpenHashMap n = getTransitionMap(language, tag, label, otherLanguage);
		double num = n.get(otherTag);
		Long2DoubleOpenHashMap q = getTransitionNormalizerMap(language, tag);
		double div = q.get(IntegerCombine.combine(label, otherLanguage));
		return amount*(Math.log(num)-Math.log(div));
	}
	/**
	 * 											
	 *
	 * @param language
	 * @param tag
	 * @param label
	 * @param otherLanguage
	 * @param otherTag
	 * @param amount
	 */
	private void addTransition(int language, int tag, int label, int otherLanguage, int otherTag,
																double amount)
	{
		Int2DoubleOpenHashMap n = this.getTransitionMap(language, tag, label, otherLanguage);
		n.addTo(otherTag, amount);
		Long2DoubleOpenHashMap q = this.getTransitionNormalizerMap(language, tag);
		q.addTo(IntegerCombine.combine(label, otherLanguage), amount);
	}
	@Override
	protected void initializeConnections(Word w, int label, Word other, double amount) 
	{
		for(int tag=0;tag<this.getNumberOfLabels(w);++tag)
		{
			int language = w.getLanguage();
			for(int otherTag=0;otherTag<this.getNumberOfLabels(other);++otherTag)
			{
				int otherLanguage = other.getLanguage();
				Int2DoubleOpenHashMap n = this.getTransitionMap(language, tag, label, otherLanguage);
				double num = n.get(otherTag);
				if(num < this.transitionSmoothing)
				{
					this.addTransition(language, tag, label, otherLanguage,
								otherTag, this.transitionSmoothing);
				}
				else
				{return;}
			}
		}
	}
	/**
	 * 
	 * @param language
	 * @param tag
	 * @return
	 */
	private Long2DoubleOpenHashMap getTransitionNormalizerMap(int language,
			int tag)
	{return getSecure(IntegerCombine.combine(language, tag), l2t2ty2l);}
	/**
	 * 
	 * @param language
	 * @param tag
	 * @param label
	 * @param otherLanguage
	 * @return
	 */
	private Int2DoubleOpenHashMap getTransitionMap(int language, int tag,
			int label, int otherLanguage)
	{return getSecure(IntegerCombine.combine(language, tag), IntegerCombine.combine(label, otherLanguage), l2t2ty2l2t);}
	/* (non-Javadoc)
	 * @see de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.model.GenericModel#addConnections(de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.data.Word, int, int, de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.data.Word, int, double, double)
	 */
	@Override
	protected void addConnections(Word w, int choice, int label, Word other,
			int otherTag, double amount, double multiplier)
	{this.addTransition(w.getLanguage(), choice, label, other.getLanguage(), otherTag, amount*multiplier);}
	/* (non-Javadoc)
	 * @see de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.model.GenericModel#makeConnectionsLogProb(de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.data.Word, int, int, de.uni_leipzig.informatik.asv.unsupervised_pos_tagger.data.Word, int, double)
	 */
	@Override
	protected double makeConnectionsLogProb(Word w, int choice, int label,
			Word other, int otherTag, double amount)
	{return this.getTransitionWeight(w.getLanguage(), choice, label,
													other.getLanguage(), otherTag, amount);}
	/**
	 *
	 * @param code1
	 * @param code2
	 * @param l2t2ty2l2t
	 * @return
	 */
	private Int2DoubleOpenHashMap getSecure(long code1,long code2,
			Long2ObjectOpenHashMap<Long2ObjectOpenHashMap<Int2DoubleOpenHashMap>> l2t2ty2l2t)
	{
		Long2ObjectOpenHashMap<Int2DoubleOpenHashMap> m = getSecure(l2t2ty2l2t, code1, true);
		Int2DoubleOpenHashMap n = this.getSecure(m, code2);
		return n;
	}
	/**
	 *
	 * @param code1
	 * @param l
	 * @return
	 */
	private Long2DoubleOpenHashMap getSecure(long code1,
			Long2ObjectOpenHashMap<Long2DoubleOpenHashMap> l)
	{
		Long2DoubleOpenHashMap q = l.get(code1);
		if(q == null)
		{
			q = new Long2DoubleOpenHashMap();
			l.put(code1, q);
		}
		return q;
	}
	/**
	 *
	 * @param l
	 * @param code1
	 * @return
	 */
	private Long2ObjectOpenHashMap<Int2DoubleOpenHashMap> getSecure(
			Long2ObjectOpenHashMap<Long2ObjectOpenHashMap<Int2DoubleOpenHashMap>> l,
			long code1, boolean dummy)
	{
		Long2ObjectOpenHashMap<Int2DoubleOpenHashMap> m = l.get(code1);
		if(m == null)
		{
			m = new Long2ObjectOpenHashMap<>();
			l.put(code1, m);
		}
		return m;
	}
}